dat_o2 <- read.csv("datall.csv", header=T)########
#dat_co2 = dat_co2[(1 + 10*(i-1)):(10*i), ]
#rownmames(dat_co2)= NULL
#length(unique(dat_co2$StudyID))
#dat_co2$y <- dat_co2$ETCO2
#dat_co2$X <- 1:nrow(dat_co2)
source('Functions.R')
load('errlist_o2_k10_n_2.Rdata')
errlist102 <- errlist

load('errlist_o2_k10_n_3.Rdata')
errlist103 <- errlist


load('errlist_o2_k15_n_2.Rdata')
errlist152 <- errlist

load('errlist_o2_k15_n_3.Rdata')
errlist153 <- errlist



load('errlist_o2_k20_n_2.Rdata')
errlist202 <- errlist

load('errlist_o2_k20_n_3.Rdata')
errlist203 <- errlist


load('errlist_o2_k25_n_2.Rdata')
errlist252 <- errlist

load('errlist_o2_k25_n_3.Rdata')
errlist253 <- errlist



load('errlist_o2_k30_n_2.Rdata')
errlist302 <- errlist

load('errlist_o2_k30_n_3.Rdata')
errlist303 <- errlist
#for (i in 1:length(unique(dat_co2$StudyID))) {
#  try(plot(errlist[[i]]$lno_grid, errlist[[i]]$err, type = 's', xlab = 'window length', ylab = 'reduced error'))
#}
l102 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist102[[i]]$lno_grid[which.max(errlist102[[i]]$err)]
  b <- tryCatch({max(errlist102[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l102 <- rbind(l102, data.frame(lno = a,error = b))
}

l102$k = 10
l102$n = 2

l103 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist103[[i]]$lno_grid[which.max(errlist103[[i]]$err)]
  b <- tryCatch({max(errlist103[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l103 <- rbind(l103, data.frame(lno = a,error = b))
}

l103$k = 10
l103$n = 3



l152 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist152[[i]]$lno_grid[which.max(errlist152[[i]]$err)]
  b <- tryCatch({max(errlist152[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l152 <- rbind(l152, data.frame(lno = a,error = b))
}

l152$k = 15
l152$n = 2

l153 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist153[[i]]$lno_grid[which.max(errlist153[[i]]$err)]
  b <- tryCatch({max(errlist153[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l153 <- rbind(l153, data.frame(lno = a,error = b))
}

l153$k = 15
l153$n = 3





l202 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist202[[i]]$lno_grid[which.max(errlist202[[i]]$err)]
  b <- tryCatch({max(errlist202[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l202 <- rbind(l202, data.frame(lno = a,error = b))
}

l202$k <- 20
l202$n = 2

l203 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist203[[i]]$lno_grid[which.max(errlist203[[i]]$err)]
  b <- tryCatch({max(errlist203[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l203 <- rbind(l203, data.frame(lno = a,error = b))
}

l203$k <- 20
l203$n = 3




l252 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist252[[i]]$lno_grid[which.max(errlist252[[i]]$err)]
  b <- tryCatch({max(errlist252[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l252 <- rbind(l252, data.frame(lno = a,error = b))
}

l252$k = 15
l252$n = 2

l253 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist253[[i]]$lno_grid[which.max(errlist253[[i]]$err)]
  b <- tryCatch({max(errlist253[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l253 <- rbind(l253, data.frame(lno = a,error = b))
}

l253$k = 15
l253$n = 3






l302 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist302[[i]]$lno_grid[which.max(errlist302[[i]]$err)]
  b <- tryCatch({max(errlist302[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l302 <- rbind(l302, data.frame(lno = a,error = b))
}

l302$k <- 30
l302$n = 2

l303 <- NULL

for (i in 1:length(unique(dat_o2$StudyID))) {
  a <- errlist303[[i]]$lno_grid[which.max(errlist303[[i]]$err)]
  b <- tryCatch({max(errlist303[[i]]$err)}, error = function(x)NA )
  a <- ifelse(length(a) == 0, yes = NA, no = a)
  l303 <- rbind(l303, data.frame(lno = a,error = b))
}

l303$k <- 30
l303$n = 3

df_err <- cbind( l102$error, l103$error, l152$error, l153$error, l202$error, l203$error, l252$error, l253$error, l302$error, l303$error)


ind_list <- do.call(c, apply(df_err, 1, which.max))

l_list <- list( l102, l103, l152, l153, l202, l203, l252, l253, l302, l303  )
i = 1

k <- unique((l_list[[ind_list[i]]])$k)

n <- unique((l_list[[ind_list[i]]])$n)

lno <- (l_list[[ind_list[i]]])$lno[i]




plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na', trim = 0.66)
plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'weighted')


plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'fitted')


plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'predicted')


plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na.weighted')


plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na.fitted')


plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na.predicted')
loop <- (1:length(unique(dat_o2$StudyID))) 

#out_indx <- c (24, 25, 30, 36, 80, 81, 85, 105, 110, 125, 138, 147, 154, 156, 178, 181, 186, 187, 188, 193, 201, 207, 208, 210, 213, 234, 239, 262, 263)
#out_indx <- c(80, 145,159,187, 253, 262, 263)
out_indx <- c (25, 36, 42, 55, 57, 80, 88, 105, 110, 115, 187, 188, 193, 198, 203, 207, 208, 210, 234, 236, 253, 262)

loop <- loop[-out_indx]

for (i in loop) {

  print( paste('Patient No.', i) )

  k <- unique((l_list[[ind_list[i]]])$k)

  n <- unique((l_list[[ind_list[i]]])$n)

  lno <- (l_list[[ind_list[i]]])$lno[i]
  
  try( plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na', trim = 0.66)  )
  lno <- ceiling(dim(dat_o2[dat_o2$StudyID == unique(dat_o2$StudyID)[i],])[1] / lno)
  legend('bottomright', paste0('window length=', lno ) )
  

}
## [1] "Patient No. 1"

## [1] "Patient No. 2"

## [1] "Patient No. 3"

## [1] "Patient No. 4"

## [1] "Patient No. 5"

## [1] "Patient No. 6"
## [1] "Patient No. 7"

## [1] "Patient No. 8"

## [1] "Patient No. 9"

## [1] "Patient No. 10"

## [1] "Patient No. 11"

## [1] "Patient No. 12"

## [1] "Patient No. 13"

## [1] "Patient No. 14"

## [1] "Patient No. 15"

## [1] "Patient No. 16"

## [1] "Patient No. 17"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n,  : 
##   no loop for break/next, jumping to top level
## [1] "Patient No. 18"

## [1] "Patient No. 19"

## [1] "Patient No. 20"

## [1] "Patient No. 21"

## [1] "Patient No. 22"

## [1] "Patient No. 23"

## [1] "Patient No. 24"
## [1] "Patient No. 26"

## [1] "Patient No. 27"

## [1] "Patient No. 28"

## [1] "Patient No. 29"

## [1] "Patient No. 30"

## [1] "Patient No. 31"

## [1] "Patient No. 32"

## [1] "Patient No. 33"

## [1] "Patient No. 34"

## [1] "Patient No. 35"

## [1] "Patient No. 37"

## [1] "Patient No. 38"

## [1] "Patient No. 39"

## [1] "Patient No. 40"

## [1] "Patient No. 41"

## [1] "Patient No. 43"

## [1] "Patient No. 44"

## [1] "Patient No. 45"

## [1] "Patient No. 46"

## [1] "Patient No. 47"

## [1] "Patient No. 48"

## [1] "Patient No. 49"

## [1] "Patient No. 50"

## [1] "Patient No. 51"

## [1] "Patient No. 52"

## [1] "Patient No. 53"

## [1] "Patient No. 54"

## [1] "Patient No. 56"

## [1] "Patient No. 58"

## [1] "Patient No. 59"
## [1] "Patient No. 60"

## [1] "Patient No. 61"

## [1] "Patient No. 62"

## [1] "Patient No. 63"

## [1] "Patient No. 64"

## [1] "Patient No. 65"

## [1] "Patient No. 66"

## [1] "Patient No. 67"

## [1] "Patient No. 68"

## [1] "Patient No. 69"

## [1] "Patient No. 70"

## [1] "Patient No. 71"

## [1] "Patient No. 72"

## [1] "Patient No. 73"

## [1] "Patient No. 74"

## [1] "Patient No. 75"

## [1] "Patient No. 76"

## [1] "Patient No. 77"
## Error in if (lno == dim(dati)[1]) { : 
##   missing value where TRUE/FALSE needed
## [1] "Patient No. 78"

## [1] "Patient No. 79"

## [1] "Patient No. 81"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n,  : 
##   no loop for break/next, jumping to top level
## [1] "Patient No. 82"

## [1] "Patient No. 83"

## [1] "Patient No. 84"

## [1] "Patient No. 85"
## Error in if (ind_begin < ind_end) { : argument is of length zero
## [1] "Patient No. 86"

## [1] "Patient No. 87"

## [1] "Patient No. 89"

## [1] "Patient No. 90"

## [1] "Patient No. 91"

## [1] "Patient No. 92"

## [1] "Patient No. 93"

## [1] "Patient No. 94"

## [1] "Patient No. 95"

## [1] "Patient No. 96"

## [1] "Patient No. 97"

## [1] "Patient No. 98"

## [1] "Patient No. 99"

## [1] "Patient No. 100"

## [1] "Patient No. 101"

## [1] "Patient No. 102"

## [1] "Patient No. 103"

## [1] "Patient No. 104"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n,  : 
##   no loop for break/next, jumping to top level
## [1] "Patient No. 106"

## [1] "Patient No. 107"

## [1] "Patient No. 108"

## [1] "Patient No. 109"
## [1] "Patient No. 111"

## [1] "Patient No. 112"

## [1] "Patient No. 113"

## [1] "Patient No. 114"

## [1] "Patient No. 116"

## [1] "Patient No. 117"

## [1] "Patient No. 118"

## [1] "Patient No. 119"

## [1] "Patient No. 120"

## [1] "Patient No. 121"

## [1] "Patient No. 122"

## [1] "Patient No. 123"

## [1] "Patient No. 124"

## [1] "Patient No. 125"

## [1] "Patient No. 126"

## [1] "Patient No. 127"

## [1] "Patient No. 128"

## [1] "Patient No. 129"

## [1] "Patient No. 130"

## [1] "Patient No. 131"

## [1] "Patient No. 132"

## [1] "Patient No. 133"

## [1] "Patient No. 134"

## [1] "Patient No. 135"
## [1] "Patient No. 136"
## [1] "Patient No. 137"

## [1] "Patient No. 138"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n,  : 
##   no loop for break/next, jumping to top level
## [1] "Patient No. 139"

## [1] "Patient No. 140"

## [1] "Patient No. 141"
## [1] "Patient No. 142"

## [1] "Patient No. 143"

## [1] "Patient No. 144"

## [1] "Patient No. 145"

## [1] "Patient No. 146"

## [1] "Patient No. 147"
## [1] "Patient No. 148"

## [1] "Patient No. 149"

## [1] "Patient No. 150"

## [1] "Patient No. 151"

## [1] "Patient No. 152"

## [1] "Patient No. 153"

## [1] "Patient No. 154"
## [1] "Patient No. 155"

## [1] "Patient No. 156"

## [1] "Patient No. 157"

## [1] "Patient No. 158"

## [1] "Patient No. 159"

## [1] "Patient No. 160"

## [1] "Patient No. 161"

## [1] "Patient No. 162"

## [1] "Patient No. 163"

## [1] "Patient No. 164"

## [1] "Patient No. 165"

## [1] "Patient No. 166"

## [1] "Patient No. 167"

## [1] "Patient No. 168"

## [1] "Patient No. 169"

## [1] "Patient No. 170"

## [1] "Patient No. 171"

## [1] "Patient No. 172"
## [1] "Patient No. 173"

## [1] "Patient No. 174"
## [1] "Patient No. 175"

## [1] "Patient No. 176"

## [1] "Patient No. 177"

## [1] "Patient No. 178"

## [1] "Patient No. 179"

## [1] "Patient No. 180"

## [1] "Patient No. 181"
## Error in plt_outlier_removal(data = dat_o2, co2 = F, i = i, k = k, n = n,  : 
##   no loop for break/next, jumping to top level
## [1] "Patient No. 182"

## [1] "Patient No. 183"
## Error in if (ind_begin < ind_end) { : argument is of length zero
## [1] "Patient No. 184"

## [1] "Patient No. 185"

## [1] "Patient No. 186"
## [1] "Patient No. 189"

## [1] "Patient No. 190"

## [1] "Patient No. 191"

## [1] "Patient No. 192"

## [1] "Patient No. 194"

## [1] "Patient No. 195"

## [1] "Patient No. 196"

## [1] "Patient No. 197"

## [1] "Patient No. 199"

## [1] "Patient No. 200"

## [1] "Patient No. 201"
## [1] "Patient No. 202"

## [1] "Patient No. 204"

## [1] "Patient No. 205"

## [1] "Patient No. 206"

## [1] "Patient No. 209"

## [1] "Patient No. 211"

## [1] "Patient No. 212"

## [1] "Patient No. 213"
## [1] "Patient No. 214"

## [1] "Patient No. 215"

## [1] "Patient No. 216"

## [1] "Patient No. 217"

## [1] "Patient No. 218"

## [1] "Patient No. 219"
## [1] "Patient No. 220"
## [1] "Patient No. 221"

## [1] "Patient No. 222"

## [1] "Patient No. 223"
## Error in if (ind_begin < ind_end) { : argument is of length zero
## [1] "Patient No. 224"

## [1] "Patient No. 225"

## [1] "Patient No. 226"

## [1] "Patient No. 227"

## [1] "Patient No. 228"

## [1] "Patient No. 229"

## [1] "Patient No. 230"

## [1] "Patient No. 231"

## [1] "Patient No. 232"

## [1] "Patient No. 233"

## [1] "Patient No. 235"

## [1] "Patient No. 237"

## [1] "Patient No. 238"

## [1] "Patient No. 239"
## [1] "Patient No. 240"

## [1] "Patient No. 241"

## [1] "Patient No. 242"

## [1] "Patient No. 243"

## [1] "Patient No. 244"

## [1] "Patient No. 245"

## [1] "Patient No. 246"

## [1] "Patient No. 247"

## [1] "Patient No. 248"

## [1] "Patient No. 249"

## [1] "Patient No. 250"

## [1] "Patient No. 251"

## [1] "Patient No. 252"

## [1] "Patient No. 254"

## [1] "Patient No. 255"

## [1] "Patient No. 256"

## [1] "Patient No. 257"
## [1] "Patient No. 258"

## [1] "Patient No. 259"

## [1] "Patient No. 260"

## [1] "Patient No. 261"

## [1] "Patient No. 263"

## [1] "Patient No. 264"

## [1] "Patient No. 265"

## [1] "Patient No. 266"

## [1] "Patient No. 267"

## [1] "Patient No. 268"

## [1] "Patient No. 269"

## [1] "Patient No. 270"

## [1] "Patient No. 271"
## Error in if (lno == dim(dati)[1]) { : argument is of length zero
loop <- 1:length(unique(dat_o2$StudyID))
#loop <- loop[!is.na(l)]
#loop <- loop[1]
clnum <- detectCores()
cl <- makeCluster(clnum)
registerDoSNOW(cl)
pb <- txtProgressBar(max = length(loop), style = 3)
progress <- function(n) setTxtProgressBar(pb, n)
opts <- list(progress = progress)
t1 <- Sys.time()
dat_o2_reduction <- foreach(x=loop,.packages=c('chron', 'refund', 'mgcv', 'fda', 'tidyverse'), .options.snow = opts, .combine = rbind ) %dopar% {
  
  k <- unique((l_list[[ind_list[x]]])$k)

  n <- unique((l_list[[ind_list[x]]])$n)

  lno <- (l_list[[ind_list[x]]])$lno[x]

  tryCatch({
   plt_outlier_removal(data = dat_o2, co2 = F, i = x, k = k, n = n, lno = lno, d = 'all', proplno = T, consecutive = T, imputation = 'na', trim = 0.6)
  }, error = function(x)NA)
  
}
  
  
  
close(pb)
stopCluster(cl)

t2 <- Sys.time()
t2 - t1




#dat_o2_reduction$y <- as.numeric(dat_o2_reduction$y)
#dat_o2_reduction$survival <- as.numeric(dat_o2_reduction$survival)


write.csv(dat_o2_reduction, file = 'dat_o2_na_29_Apr_2020.csv')